#!/usr/bin/env python
#-*- coding:utf-8 -*-

#*************************
# FileName:WordFilter.py
# DateTime:2008/11/17
# Author:wule
#*************************

import time
import sys
from os import path

try:
    import psyco
    psyco.full()
except ImportError:pass

KEYWORD_FILE_CODING = 'UTF-8'
_CHART_SYMBOL = [u' ', u'\t', u'\n', u'\r', u',', u'<', u'>', u'?', u'!', u';', 
                u'#', u':', u'.', u"'", u'"', u'(', u')', u'{', u'}', u'[', u']',
                u'|', u'_', u'=', u"　", u"；", u"，", u"？", u"。", u"、", u"“", 
                u"”", u"《", u"》", u"［", u"］", u"！", u"（", u"）", u"—", u"…"]

#-------------------------------------------------------------------------------
class FilterTest(object):
    WTYPE_BAD_WORD = 1    #这个是只出现一次就要屏蔽的坏词
    WTYPE_WORST_WORD = 2  #这个是要出现累计次数才屏蔽的词
    SKIP_SYMBOL = dict(map(lambda x:(x, None), _CHART_SYMBOL))
    #add more word filter type
    def __init__(self):
        self.MAX_LENGTH = -1       #keyword中最长的词的长度
        self.RootPath = path.dirname(path.abspath(sys.argv[0]))
        self.KeyWords = {}
        self.EnglistKeyWords = {}
        self.init()
        
    def init(self):
        FBAD_WORD = path.join(self.RootPath, 'bad_words.txt')
        FWORST_WORD = path.join(self.RootPath, 'worst_words.txt')        
        self.load(FBAD_WORD, KEYWORD_FILE_CODING, self.WTYPE_BAD_WORD)
        self.load(FWORST_WORD, KEYWORD_FILE_CODING, self.WTYPE_WORST_WORD)
        #add more filter keyword file load in here
        
    def load(self, keywordFile, coding='utf-8', wtype=1):
        try:
            KF = open(keywordFile, 'rb')
            for line in KF.xreadlines():
                try:
                    line = line.strip().decode(coding, 'ignore')
                    if line:
                        self.KeyWords[line] = wtype
                        klength = len(line)
                        self.MAX_LENGTH = klength if klength > self.MAX_LENGTH else self.MAX_LENGTH
                except:pass
        except Exception, e:
            raise Exception("load File Error.(%s)" % str(e))
        
    def toUnicode(self, text, coding='utf-8'):
        if text:
            return text.decode(coding, 'ignore')
        else:
            return u''    
    
    def getAllKeywords(self, shtml, coding='utf-8', limitCount=3):
        shtml = self.toUnicode(shtml, coding)
        shtml_length = len(shtml)
        result_dict = {}
        #java版本里用的 是类似正向最大匹配的方法 
        #这里做了点点改动 解决漏掉那种短词问题
        for i in xrange(shtml_length):
            try:
                if self.SKIP_SYMBOL.has_key(shtml[i]):
                    #如果碰到标点符号 直接跳过
                    continue
                for j in xrange(1, self.MAX_LENGTH + 1):
                    m = i + j
                    m = shtml_length if m > shtml_length else m
                    word = shtml[i:m]
                    if self.KeyWords.has_key(word):
                        #key 为词的unicode 切片位置
                        #如 32_36的词为shtml[32:36]
                        #value 第一个为词的类型 第二个为 词
                        result_dict['%d_%d' % (i, m)] = (self.KeyWords.get(word), word)
            except Exception, e:
                print "[%s] %s\n" % (time.strftime("%Y-%m-%d %H:%M:%S"), str(err))
        if len(result_dict) > limitCount - 1:
            return (False, result_dict)
        elif len(result_dict) > 0:
            for r in result_dict.values():
                if r[0] == 1:
                    return (False, result_dict)
            return (True, result_dict)
        else:
            return (True, result_dict)

if __name__ == '__main__':
    T = FilterTest()
#    txt = open("r:\\err.txt", 'rb').read()
#    print T.getAllKeywords(txt, 'gbk')
    
    
    shtml = "齐白石 骚屄 张大钱 恶党"
#    shtml = '''  '''
    from timeit import default_timer
    ts = default_timer()
    m = T.getAllKeywords(shtml)
    print "UseTime:", default_timer() - ts
    print m
    print '**********'
    for i in m[1].values():
        print i[0], i[1].encode('gb2312', 'ignore')